import random
import json
import requests
from os import path
import chardet
from urllib.parse import urlparse
from bs4 import BeautifulSoup


class MyEncoder(json.JSONEncoder):

    def default(self, obj):
        """
        只要检查到了是bytes类型的数据就把它转为str类型
        :param obj:
        :return:
        """
        if isinstance(obj, bytes):
            return str(obj, encoding='utf-8')
        return json.JSONEncoder.default(self, obj)
urls=[]

with open('./was_todo_list','rb') as f:
    urls.extend(f.readlines())
    map(lambda url : url.strip(),urls)
print (urls)

with open('./template2.was','rb') as f :
    template = json.loads(f.read())
    for url in urls[9:]:
        try:
            url = str(url,encoding='gbk').strip()
            _url = urlparse(url)
            hostname = str(_url.hostname)
            x = hostname.index('.')
            y = hostname.rindex('.')
            hostname = hostname[x+1:y]
            file_to_write = './gen/' + str(hostname) +'.was'
            if(path.exists(file_to_write)):
                print('exists continue')
                continue
            print('cur url is ', url)
            r = requests.get(url.strip(),timeout=30)
            text = r.text
            # print(charset)
            # print(r.status_code)
            charset1 = chardet.detect(r.content)
            # print(charset1)
            soup = BeautifulSoup(r.content,'html.parser',from_encoding=charset1['encoding'])
            title_str = soup.title.text
            title_str = title_str[:7]
            if(title_str.find(' ') > 0):
                title_str=title_str[:title_str.find(' ')]
            elif(title_str.find('_') > 0):
                title_str=title_str[:title_str.find('_')]
            elif(title_str.find('-') > 0):
                title_str=title_str[:title_str.find('-')]
            else:
                title_str = url

            template['siteName'] = str(title_str)
            print(title_str)
            template['siteHostUrl'] = str(url)
            template['priority'] = random.randint(23,59)

            with open ('./gen/' + str(hostname) +'.was', 'w', encoding='utf-8') as fw:
                fw.write(json.dumps(template, sort_keys=True, indent=4, cls=MyEncoder, ensure_ascii=False))
        except Exception as e:
            # import traceback
            # traceback.print_exc(e)
            print('出错了'+e.__str__())
            continue
